package com.xl.competition.modul_b.task2

import org.apache.spark.sql.expressions.UserDefinedFunction
import org.apache.spark.sql.functions.{col, udf}
import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession}

import java.text.SimpleDateFormat
import java.util.{Date, Properties}

/**
 * @author: xl
 * @createTime: 2023/12/22 20:33:13
 * @program: com.xl.competition
 * @description: ${description}
 */
object TEst {
  def main(args: Array[String]): Unit = {
    val spark: SparkSession = SparkSession.builder()
      .master("local[*}")
      .appName("test")
      .enableHiveSupport()
      .enableHiveSupport()
      .config("hive.metastore.uris", "thrift://node2:9083")
      .config("spark.sql.parquet.writeLegacyFormat", "true")
      .getOrCreate()


    // spark core   spark sql== >    { 1. sql      2.dsl}
    val prop = new Properties()
    prop.put("user", "root")
    prop.put("password", "Abc123..")
    val df: DataFrame = spark.read
      .jdbc("jdbc:mysql://node3:3306/shtd_store", "order_info", prop)

    val row: Row = spark.sql(
      """
        |select max(id) from ods.base_province
        |""".stripMargin)
      .take(1)(0)

    var id: Long = 0
    if (!row.isNullAt(0)) {
      id = row.getLong(0)
    }


    val create_udf: UserDefinedFunction = udf(() => new SimpleDateFormat("yyyy-mm-dd HH:mm:SS").format(new Date))
    val etl_date: UserDefinedFunction = udf(() => "20231222")



    df.where(col("id") > id)
      .withColumn("create_time", create_udf())
      .withColumn("etl_date", etl_date())
      .select("id", "name", "region_id", "area_code", "iso_code", "iso_3166_2", "create_time")
      .write
      .mode(SaveMode.Append)
      .format("hive")
      .partitionBy("etl_date")
      .save()

    spark.stop()
  }

}
